package service;

import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.HashSet;

import org.apache.log4j.Logger;

import common.Constants;

import crawler.CrawlerDocument;

public class Deduper implements Runnable {
	
	private final Logger logger = Logger.getLogger(Deduper.class);
	
	private static Deduper self = null;
	private static MessageDigest _md5Maker = null;
	
	private HashSet<String> _contentMd5 = null; 
	
	private Deduper(){
		_contentMd5 = new HashSet<String>();
		try {
			_md5Maker = MessageDigest.getInstance("MD5");
		} catch (NoSuchAlgorithmException e) {
			logger.error("{}", e);
		}
	}
	
	public static Deduper getInstance(){
		if (null == self){
			self = new Deduper();
		}
		if (null == _md5Maker){
			return null;
		}else {
			return self;
		}
	}
	
	public boolean isDuplicated(CrawlerDocument doc){
		String link = (String)doc.getFeature(Constants.XPATH_LINK);

		_md5Maker.update(link.getBytes());
		byte[] md5 = _md5Maker.digest(); 
		String md5Str = bytesToHex(md5);
		
		logger.trace("doc md5 sum to dedupe : " + md5Str);
		
		boolean docExist = !_contentMd5.add(md5Str);
		
		logger.debug("Is link [ " + link + "] duplicate : " + docExist);
		
		return docExist;
	}
	
	private static String bytesToHex(byte[] b) {
		char hexDigit[] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
				'A', 'B', 'C', 'D', 'E', 'F' };
		StringBuffer buf = new StringBuffer();
		for (int j = 0; j < b.length; j++) {
			buf.append(hexDigit[(b[j] >> 4) & 0x0f]);
			buf.append(hexDigit[b[j] & 0x0f]);
		}
		return buf.toString();
	}	

	@Override
	public void run() {
		// TODO Persistence the data.
		
	}

}
